from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
# import gc
import copy
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
# import numpy as np
from os import walk
# import matplotlib.pyplot as plt
from subprocess import call
import os
import collections

if len(sys.argv) == 3:
    # arg1: the posting beginning position
    # arg2: the document posting array file
    beginningPosition = int(sys.argv[1])
    fileName = sys.argv[2]
else:
    print "USAGE: programName.py beginningPosition fileName"
    exit(1)

# some numbers:
#        0            0
#  1000000  20407812012 
#  2000000  37979581056  
#  3000000  53446024716
#  4000000  65582188956
#  5000000  75814235460
#  6000000  85833072684
#  7000000  94580553876
#  8000000  102778986396
#  9000001  108537313656
#  10000001 113048112924
# 11000000 117472283112
# 12000002 121894212264
# 13000000 126261377304
# 14000003 130280992908
# 15000001 133414035600
# 16000000 136566045192
# 17000000 139463385108
# 18000002 142330945980
# 19000005 145112762640
# 20000001 147622084956
# 21000002 150042023376
# 22000000 165315614076
# 23000000 187715366112
# 24000000 207087755040
# 25000000 232181415996

print "Begin..."
previousTermID = 0
previousPostingHitFreq = 0

# step0:
debugCounter = 0
docIDAndTOP10HitDict = {}
ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_final_sortedByDocID_withNumOfPostingsAdded"
ifh = open(ifn0,"r")
l = ifh.readline()
while l:
    le = l.strip().split(" ")
    docID = int(le[0])
    numOfDocHits = int(le[1])
    docIDAndTOP10HitDict[docID] = numOfDocHits 
    l = ifh.readline()
    if debugCounter % 100000 == 0:
        print debugCounter,"processed."
    debugCounter += 1
    #if debugCounter == 10:
    #    break
print "Overall:"
print "len(docIDAndTOP10HitDict):",len(docIDAndTOP10HitDict)
print "docIDAndTOP10HitDict[3]:",docIDAndTOP10HitDict[3]
ifh.close()

# step1:
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_50M/50MQueries_TOP1000/binaryFiles/resultfile0_50MQueries_postingHits_20141106_final.binary0"
inputFileHandler0 = open(ifn,"rb")
statinfo = os.stat(ifn)
fileSize = statinfo.st_size
print "currentInputFileName: ",ifn
print "file size:",fileSize
numOfBytesRead = 0

# ofn = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_docHit_postingHit_0M_1M"
# ofn = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_docHit_postingHit_1M_2M"
ofn = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/" + fileName + "_docHit_postingHit_added"
ofh = open(ofn,"w")

# ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_0M_1M"
# ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_1M_2M"
ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/" + fileName
inputFileHandler1 = open(ifn1,"rb")

statinfo1 = os.stat(ifn1)
fileSize1 = statinfo1.st_size
print "currentInputFileName: ",ifn1
print "file size1:",fileSize1
numOfBytesRead1 = 0
numOfBytesWritten1 = 0
numOfDocumentsProcessed1 = 0
indepentBytesChecker = 0

# init
currentDocID = -1
currentTermIDDict = {}
inputFileHandler0.seek(beginningPosition)
numOfBytesRead = beginningPosition
while numOfBytesRead < fileSize:
    byteString = inputFileHandler0.read(4 + 4 + 4)
    (docID,termID,postingHitFreq) = unpack( "3I", byteString)
    # print docID,termID,postingHitFreq
    
    if docID != currentDocID:
        # output the thing
        if currentDocID != -1:
            previousTermID = termID
            previousPostingHitFreq = postingHitFreq
            # print currentDocID,currentTermIDDict
            break
        
        # init the variables
        currentTermIDDict = {}
        currentDocID = docID
        if termID not in currentTermIDDict:
            currentTermIDDict[termID] = postingHitFreq
        else:
            currentTermIDDict[termID] += postingHitFreq
    else:
        if termID not in currentTermIDDict:
            currentTermIDDict[termID] = postingHitFreq
        else:
            currentTermIDDict[termID] += postingHitFreq
    numOfBytesRead += 12
# print currentDocID,currentTermIDDict

while numOfBytesRead1 < fileSize1:
    byteString = inputFileHandler1.read(4 + 4)
    (docIDFromFile,numOfPostings) = unpack( "2I", byteString)
    
    #if docIDFromFile == 10000010:
    #    print "indepentBytesChecker: ",indepentBytesChecker
    #    break
    
    indepentBytesChecker += 12 + numOfPostings * 12
    
    if docIDFromFile in docIDAndTOP10HitDict:
        ofh.write(pack("3I",docIDFromFile,numOfPostings,docIDAndTOP10HitDict[docIDFromFile]))
    else:
        ofh.write(pack("3I",docIDFromFile,numOfPostings,0))
    print "docID:",docIDFromFile,numOfPostings
    needToLookUpFlag = False
    newFillUpFlag = False
    if docIDFromFile == currentDocID:
        needToLookUpFlag = True
        newFillUpFlag = True
    
    for i in range(0,numOfPostings):
        byteString = inputFileHandler1.read(4 + 4 + 4 + 4 + 4)
        (termID,_,_,_,impactScore) = unpack( "1I4f", byteString)
        if needToLookUpFlag:
            if termID in currentTermIDDict:
                ofh.write(pack("1I1f1I",termID,impactScore,currentTermIDDict[termID]))
            else:
                ofh.write(pack("1I1f1I",termID,impactScore,0))
        else:
            ofh.write(pack("1I1f1I",termID,impactScore,0))
    
    if newFillUpFlag:
        currentDocID = -1
        currentTermIDDict = {}
        while numOfBytesRead < fileSize:
            byteString = inputFileHandler0.read(4 + 4 + 4)
            (docID,termID,postingHitFreq) = unpack( "3I", byteString)
            # print docID,termID,postingHitFreq
            
            if docID != currentDocID:
                
                if currentDocID != -1:
                    previousTermID = termID
                    previousPostingHitFreq = postingHitFreq
                    # print currentDocID,currentTermIDDict
                    break
                
                # init the variables
                currentTermIDDict = {}
                currentTermIDDict[previousTermID] = previousPostingHitFreq
                currentDocID = docID
                
                if termID not in currentTermIDDict:
                    currentTermIDDict[termID] = postingHitFreq
                else:
                    currentTermIDDict[termID] += postingHitFreq
            else:
                
                if termID not in currentTermIDDict:
                    currentTermIDDict[termID] = postingHitFreq
                else:
                    currentTermIDDict[termID] += postingHitFreq
                
            numOfBytesRead += 12
        
        needToLookUpFlag = False
        newFillUpFlag = False
    
    numOfBytesRead1 += 8 + numOfPostings * 20
    numOfBytesWritten1 += 12 + numOfPostings * 12
inputFileHandler1.close()
ofh.close()
inputFileHandler0.close()

print "Overall:"
print "ifn0:",ifn0
print "ifn:",ifn
print "ifn1:",ifn1
print "ofn:",ofn
print "numOfBytesWritten1:",numOfBytesWritten1
print "Ends."

















